'''
Created on 4 avr. 2013

@author: Nils Amiet
'''

# coding: utf-8

graph1Path = "graph1.gml"
graph2Path = "graph2.gml"

rankingsFile = "rankings.html"

import networkx as nx
import networkx.readwrite.gml as gml
import operator
import math
import copy
import os
import sys
import sqlite3

''' Required for running the script from anywhere outside eclipse'''
os.chdir(os.path.dirname(os.path.realpath(__file__)))
sys.path.append('../../../../../InfrarougeGraphBuilder')

from ch.hearc.infrarouge.graphbuilder.HTMLTableGenerator import HTMLTableGenerator
from ch.hearc.infrarouge.graphbuilder.main import users, polarityReplies, getForums, databasePath
from ch.hearc.infrarouge.graphbuilder.polarity import PolarityCounter

sqlUsers = users()
averageRankings = []

def readGraphs():
    g1 = gml.read_gml(graph1Path, relabel=True)
    g2 = gml.read_gml(graph2Path, relabel=True)
    return (g1, g2)

def getSortedTop(centrality):
    return sorted(centrality.items(), key=operator.itemgetter(1), reverse=True)

def writeHTML(filename, headings, rows, readWriteMode='w'):
    html = HTMLTableGenerator(th=headings, rows=rows, users=sqlUsers)
    with open(filename, readWriteMode) as file:
        file.write(str(html))
        
def writeRawHTML(filename, headings, rows, readWriteMode='w'):
    htmlGenerator = HTMLTableGenerator(th=headings, rows=rows, users=sqlUsers)
    html = htmlGenerator.raw()
    with open(filename, readWriteMode) as file:
        file.write(str(html))
        
def writeHeader(filename):
    html = HTMLTableGenerator(title="Infrarouge Centrality rankings")
    with open(filename, 'w') as file:
        file.write(str(html.header()))
        
def writeFooter(filename):
    html = HTMLTableGenerator()
    with open(filename, 'a') as file:
        file.write(str(html.footer()))
        
def writeH1(filename, text):
    html = HTMLTableGenerator()
    with open(filename, 'a') as file:
        file.write(html.h1(text))

def createRanking(graph, ev=True, directed=False):
    # general graph information
    headings = ["#nodes", "#edges", "directed", "weighted", "bipartite"]
    rows = []
    if directed:
        weighted = True
        bipartite = False
    else:
        weighted = False
        bipartite = True
    rows += [(str(len(graph.nodes())), str(len(graph.edges())), str(directed), str(weighted), str(bipartite))]
    writeRawHTML(rankingsFile, headings, rows, readWriteMode='a')
    
    # centrality rankings
    degreeCentrality = nx.degree_centrality(graph)
    if directed:
        inDegreeCentrality = nx.in_degree_centrality(graph)
        outDegreeCentrality = nx.out_degree_centrality(graph)
    closenessCentrality = nx.closeness_centrality(graph)
    betweennessCentrality = nx.betweenness_centrality(graph)

    nbNodes = len(graph.nodes())
    topLength = 15
    degreeRanking = getSortedTop(degreeCentrality)
    if directed:
        inDegreeRanking = getSortedTop(inDegreeCentrality)
        outDegreeRanking = getSortedTop(outDegreeCentrality)

    closenessRanking = getSortedTop(closenessCentrality)
    betweennessRanking = getSortedTop(betweennessCentrality)
    

    headings = ["user id", "username", "degree centrality"]
    writeHTML(rankingsFile, headings, degreeRanking[:topLength], readWriteMode='a')
    
    if directed:
        headings = ["user id", "username", "in-degree centrality"]
        writeHTML(rankingsFile, headings, inDegreeRanking[:topLength], readWriteMode='a')
        
        headings = ["user id", "username", "out-degree centrality"]
        writeHTML(rankingsFile, headings, outDegreeRanking[:topLength], readWriteMode='a')
    
    headings = ["user id", "username", "closeness centrality"]
    writeHTML(rankingsFile, headings, closenessRanking[:topLength], readWriteMode='a')
    
    headings = ["user id", "username", "betweenness centrality"]
    writeHTML(rankingsFile, headings, betweennessRanking[:topLength], readWriteMode='a')
        
    if ev:
        evCentrality = nx.eigenvector_centrality(graph, max_iter=500)
        evRanking = getSortedTop(evCentrality)
            
        headings = ["user id", "username", "eigenvector centrality"]
        writeHTML(rankingsFile, headings, evRanking[:topLength], readWriteMode='a')

    clusteringCoefficientRanking = computeClusteringCoefficient(graph, directed)
    
    # clustering coefficient        
    headings = ["user id", "username", "clustering coefficient"]
    writeHTML(rankingsFile, headings, clusteringCoefficientRanking[:topLength], readWriteMode='a')
        
    # fraction of common users
    commonNodesStatsDCBE = []
    commonNodesStatsDE = []
    commonNodesStatsCE = []
    commonNodesStatsBE = []
    commonNodesStatsDC = []
    commonNodesStatsDB = []
    commonNodesStatsBC = []
    commonNodesStatsDCB = []
    
    fraction = 1
    
    while fraction <= 20: # from 1% to 20%
        if ev:
            commonNodesFractionDCBE = computeFractionOfCommonUsers(fraction/100.0, degreeRanking, closenessRanking, betweennessRanking, evRanking)
            commonNodesFractionDE = computeFractionOfCommonUsers(fraction/100.0, degreeRanking, evRanking)
            commonNodesFractionCE = computeFractionOfCommonUsers(fraction/100.0, closenessRanking, evRanking)
            commonNodesFractionBE = computeFractionOfCommonUsers(fraction/100.0, betweennessRanking, evRanking)
            
            commonNodesStatsDCBE += [(str(fraction) + "%", str(100*commonNodesFractionDCBE) + "%")]
            commonNodesStatsDE += [(str(fraction) + "%", str(100*commonNodesFractionDE) + "%")]
            commonNodesStatsCE += [(str(fraction) + "%", str(100*commonNodesFractionCE) + "%")]
            commonNodesStatsBE += [(str(fraction) + "%", str(100*commonNodesFractionBE) + "%")]
        
        commonNodesFractionDCB = computeFractionOfCommonUsers(fraction/100.0, degreeRanking, closenessRanking, betweennessRanking)
            
        commonNodesFractionDC = computeFractionOfCommonUsers(fraction/100.0, degreeRanking, closenessRanking)
        commonNodesFractionDB = computeFractionOfCommonUsers(fraction/100.0, degreeRanking, betweennessRanking)
        commonNodesFractionBC = computeFractionOfCommonUsers(fraction/100.0, betweennessRanking, closenessRanking)
        
        commonNodesStatsDC += [(str(fraction) + "%", str(100*commonNodesFractionDC) + "%")]
        commonNodesStatsDB += [(str(fraction) + "%", str(100*commonNodesFractionDB) + "%")]
        commonNodesStatsBC += [(str(fraction) + "%", str(100*commonNodesFractionBC) + "%")]
        
        commonNodesStatsDCB += [(str(fraction) + "%", str(100*commonNodesFractionDCB) + "%")]
        
        fraction += 1
        
    
    if ev:
        headings = ["top proportion", "fraction of common users [DC, CC, BC, EC]"]
        writeRawHTML(rankingsFile, headings, commonNodesStatsDCBE, readWriteMode='a')
        
    headings = ["top proportion", "fraction of common users [DC, CC, BC]"]
    writeRawHTML(rankingsFile, headings, commonNodesStatsDCB, readWriteMode='a')
    
    topNumber = "top proportion"
    if ev:
        headingsDE = [topNumber, "fraction of common users [DC, EC]"]
        headingsCE = [topNumber, "fraction of common users [CC, EC]"]
        headingsBE = [topNumber, "fraction of common users [BC, EC]"]
        
        writeRawHTML(rankingsFile, headingsDE, commonNodesStatsDE, readWriteMode='a')
        writeRawHTML(rankingsFile, headingsCE, commonNodesStatsCE, readWriteMode='a')
        writeRawHTML(rankingsFile, headingsBE, commonNodesStatsBE, readWriteMode='a')
        
    headingsDC = [topNumber, "fraction of common users [DC, CC]"]
    headingsDB = [topNumber, "fraction of common users [DC, BC]"]
    headingsBC = [topNumber, "fraction of common users [BC, CC]"]
        
    writeRawHTML(rankingsFile, headingsDC, commonNodesStatsDC, readWriteMode='a')
    writeRawHTML(rankingsFile, headingsDB, commonNodesStatsDB, readWriteMode='a')
    writeRawHTML(rankingsFile, headingsBC, commonNodesStatsBC, readWriteMode='a')
        
    # top 15 common nodes
    topn = 15
    if ev:
        fraction = topn / float(nbNodes)
        commonNodesTop15Fraction = computeFractionOfCommonUsers(fraction, degreeRanking, closenessRanking, betweennessRanking, evRanking)
        commonNodesTop15Stats = [("Top " + str(topn), str(int(commonNodesTop15Fraction*topn)) + " (" + str(100*commonNodesTop15Fraction) + "%)")]
        
        topNumber = "top number"
        headings = [topNumber, "Common users [DC, CC, BC, EC] in top " + str(topn)]
        writeRawHTML(rankingsFile, headings, commonNodesTop15Stats, readWriteMode='a')
    
    # average rank
    topNodes = []
    if ev:
        # compute average of all 4
        rankings = [degreeRanking, closenessRanking, betweennessRanking, evRanking]
    else:
        # compute average of all 3
        rankings = [degreeRanking, closenessRanking, betweennessRanking]
    
    for ranking in rankings:
        topNodes += [x[0] for x in ranking]
    topNodes = set(topNodes)
        
    averageRankStats = []
    for node in topNodes:
        averageRank = 0
        for ranking in rankings:
            try:
                rank = [x[0] for x in ranking].index(node)
            except:
                print("Warning: Node not in ranking!")
            averageRank += rank
        
        averageRank /= float(len(rankings))
        averageRank += 1
        averageRankStats += [(node, averageRank)]
        averageRankStats = sorted(averageRankStats, key=operator.itemgetter(1), reverse=False)
        
    global averageRankings
    averageRankings += [averageRankStats]
        
    headings = ["node id", "username", "average rank"]
    writeHTML(rankingsFile, headings, averageRankStats[:topn], readWriteMode='a')
    
    # top n% subgraph
    limit = 7
    for topPercent in range(10, 100, 15):
        topNumber = int(nbNodes * topPercent / float(100))
        topNodes = [x[0] for x in averageRankStats][:topNumber]
        
        topNodesAndNeighbors = copy.deepcopy(topNodes)
        for node in topNodes:
            topNodesAndNeighbors += graph.neighbors(node)
        
        subGraph = nx.subgraph(graph, topNodes)
        subGraphWithNeighbors = nx.subgraph(graph, topNodesAndNeighbors)
        
        filename = "graph" + ("1" if directed else "2") + "top" + str(topPercent) + "percent.gml"
        saveGraph(subGraph, filename)
        
        clusteringCoefficientRanking = computeClusteringCoefficient(subGraph, directed)
        
        clusteringCoefficientNeighborsRanking = computeClusteringCoefficient(subGraphWithNeighbors, directed)
        clusteringCoefficientNeighborsRanking = [x for x in clusteringCoefficientNeighborsRanking if x[0] in topNodes]
        
        headings = ["user id", "username", "clustering coefficient for top" + str(topPercent) + "% subgraph"]
        writeHTML(rankingsFile, headings, clusteringCoefficientRanking[:limit], readWriteMode='a')
        
        headings = ["user id", "username", "clustering coefficient (with neighbors) for top" + str(topPercent) + "% subgraph"]
        writeHTML(rankingsFile, headings, clusteringCoefficientNeighborsRanking[:limit], readWriteMode='a')
        
def computeClusteringCoefficient(graph, directed):
    if not directed:
        clusteringCoefficient = nx.clustering(graph)
    else:
        g = nx.Graph()
        g.add_nodes_from(graph.nodes())
        g.add_edges_from(graph.edges())
        clusteringCoefficient = nx.clustering(g)
        
    clusteringCoefficientRanking = getSortedTop(clusteringCoefficient)
    
    return clusteringCoefficientRanking
    
def saveGraph(graph, filename):
    gml.write_gml(graph, filename)
    
def computeFractionOfCommonUsers(fraction, *rankings):
    if len(rankings) > 1:
        nbNodes = len(rankings[0])
        portionSize = math.ceil(fraction * nbNodes)
        commonNodes = set([x[0] for x in rankings[0]][:portionSize])
        
        for ranking in rankings:
            rankingPortion = [x[0] for x in ranking][:portionSize]
            commonNodes = [x for x in commonNodes if x in rankingPortion]
            
        commonNodesFraction = len(commonNodes) / float(portionSize)
        return commonNodesFraction
    raise Exception("expected at least 2 rankings")

def createGeneralRanking():
    generalRanking = {}
    generalRankingNodeOccurences = {}
    for ranking in averageRankings:
        for nodeRankTuple in ranking:
            node = nodeRankTuple[0]
            rank = nodeRankTuple[1]
            try:
                generalRanking[node] += rank
            except:
                generalRanking[node] = rank
                
            try:
                generalRankingNodeOccurences[node] += 1
            except:
                generalRankingNodeOccurences[node] = 1
    
    finalRanking = []
    for node, rank in generalRanking.items():
        averageRank = rank / float(generalRankingNodeOccurences[node])
        finalRanking += [(node, averageRank)]
    
    finalRanking = sorted(finalRanking, key=operator.itemgetter(1), reverse=False)
    
    headings = ["node id", "username", "average rank over both graphs"]
    writeHTML(rankingsFile, headings, finalRanking, readWriteMode='a')
    
def createPolarityRanking():
    limit = 7
    # polarity ranking
    replies = polarityReplies()
    polarityCounter = PolarityCounter(replies)
    polarityRanking = polarityCounter.polarityDifferenceRanking()
    
    headings = ["user id", "username", "Greatest polarity difference"]
    writeHTML(rankingsFile, headings, polarityRanking[:limit], readWriteMode='a')
    
    polarityRanking.reverse()
    headings = ["user id", "username", "Lowest polarity difference"]
    writeHTML(rankingsFile, headings, polarityRanking[:limit], readWriteMode='a')
    
    print("users: %s, ranking length: %s" % (len(sqlUsers), len(polarityRanking)))
    print(len(polarityCounter.replies))
    print(len(polarityCounter.edges))
    print("NDI=" + str(polarityCounter.NDI))
    

def printTime(interval):
    seconds = interval / 1000
    minutes = seconds / 60
    hours = minutes / 60
    days = hours / 24
    print(days, hours, minutes, seconds)
    
def polarizationForAllForums():
    forums = getForums()
    
    for forum in forums:
        forumID = forum["id"]
        replies = polarityReplies(forum=forumID)
        computePolarizationOverTime(replies, forumID)

def computePolarizationOverTime(replies, forumID):
    timeSortedReplies = sorted(replies, key=lambda x: x["timestamp"])
    first = timeSortedReplies[0]["timestamp"]
    last = timeSortedReplies[-1]["timestamp"]
    interval = last - first
    timeSliceCount = 20
    deltaTimeMillis = interval / timeSliceCount
    
    ndiVariation = []
    timeThreshold = first + deltaTimeMillis
    while timeThreshold <= last:
        r = repliesBeforeTime(timeSortedReplies, timeThreshold)
        polarityCounter = PolarityCounter(r)
        edgesCount = len(polarityCounter.edges)
        ndiVariation += [(timeThreshold, polarityCounter.NDI, edgesCount)]
        
        timeThreshold += deltaTimeMillis
        
    times = [x[0] for x in ndiVariation]
    ndis = [x[1] for x in ndiVariation]
    edgesCounts = [x[2] for x in ndiVariation]
    
#    print(times)
#    print(ndis)
#    print(edgesCounts)
    
    filename = "topic" + str(forumID) + ".m"
    writeListsToFile(times, ndis, edgesCounts, filename)
    
def writeListsToFile(times, ndis, edgesCounts, filename):
    with open(filename, 'w+') as file:
        file.write("times = " + str(times) + ";\n")
        file.write("ndis = " + str(ndis) + ";\n")
        file.write("edges = " + str(edgesCounts) + ";\n")
        
        plotCode = """
figure
plot(times, ndis)
xlabel('Time (timestamp)')
ylabel('Network Disagreement Index (NDI)')
        """
        file.write(plotCode)

def repliesBeforeTime(replies, time):
    return [reply for reply in replies if reply["timestamp"] <= time]

def usersWhoSentMultipleMessages(forum=1):
    query = """
SELECT m.fkauthor, f.id, COUNT(*) as count
FROM threadmessage as m
INNER JOIN forumthread as t
    ON m.fkforumthread=t.id
INNER JOIN forum as f
    ON t.fkparentforum=f.id
WHERE f.id=%s
GROUP BY m.fkauthor
HAVING count>1
    """ % forum
    
    users = []
    
    with sqlite3.connect(databasePath) as connection:
        cursor = connection.cursor()
        cursor.execute(query)
        
        for user in cursor:
            users += [user]
        
        cursor.close()
        
    return users

def usersWhoParticipateInForum(forum=1):
    query = """
SELECT m.fkauthor, f.id, COUNT(*) as count
FROM threadmessage as m
INNER JOIN forumthread as t
    ON m.fkforumthread=t.id
INNER JOIN forum as f
    ON t.fkparentforum=f.id
WHERE f.id=%s
GROUP BY m.fkauthor
    """ % forum
    
    users = []
    
    with sqlite3.connect(databasePath) as connection:
        cursor = connection.cursor()
        cursor.execute(query)
        
        for user in cursor:
            users += [user]
        
        cursor.close()
        
    return users


def getUserCount():
    query = "SELECT COUNT(*) as count FROM user"
    with sqlite3.connect(databasePath) as connection:
        cursor = connection.cursor()
        cursor.execute(query)
        for row in cursor:
            count = row[0]
       
        cursor.close()
       
    return count

def forumContributionRanking():
    forums = getForums()
    
    ranking = []
    
    for forum in forums:
        forumID = forum["id"]
        usersWhoSentMoreThanOneMessage = usersWhoSentMultipleMessages(forum=forumID)
        users = usersWhoParticipateInForum(forum=forumID)
        gtOne = len(usersWhoSentMoreThanOneMessage)
        total = len(users)
        ratio = gtOne/float(total)*100
        line = (forumID, gtOne, total, str(ratio) + "%")
        ranking += [line]
        
    ranking = sorted(ranking, key=lambda x: x[1]/float(x[2]), reverse=True)
        
    
    headings = ["forum id", "users with sent messages count > 1", "users who sent at least 1 message count", "ratio"]
    writeRawHTML(rankingsFile, headings, ranking, readWriteMode='a')

if __name__ == "__main__":
    (g1, g2) = readGraphs()
    print(len(g1.nodes()))
    print(len(g2.nodes()))
    
    writeHeader(rankingsFile)
    
    print("-----")
    print("Centrality Rankings for graph 1")
    print("-----")
    writeH1(rankingsFile, "Replies graph centrality measures (directed weighted)")
    createRanking(g1, directed=True)
    
    print()
    print("-----")
    print("Centrality Rankings for graph 2")
    print("-----")
    writeH1(rankingsFile, "User-Discussion graph centrality measures (bipartite)")
    createRanking(g2, ev=False) # do not compute Eigenvector centrality
    
    print()
    print("-----")
    print("Conclusion")
    print("-----")
    writeH1(rankingsFile, "Conclusion")
    createGeneralRanking()
    
    createPolarityRanking()
    polarizationForAllForums()
    forumContributionRanking()
    
    writeFooter(rankingsFile)
    
    print()
    print("Successfully generated HTML report.")